{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Simple tutorial using code from the TensorFlow example for Regression.\\n\\nParag K. Mital, Jan. 2016'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"Simple tutorial using code from the TensorFlow example for Regression.\n",
    "\n",
    "Parag K. Mital, Jan. 2016\"\"\"\n",
    "# pip3 install --upgrade\n",
    "# https://storage.googleapis.com/tensorflow/mac/tensorflow-0.6.0-py3-none-any.whl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# %%\n",
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "import tensorflow.examples.tutorials.mnist.input_data as input_data\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# %%\n",
    "# get the classic mnist dataset\n",
    "# one-hot means a sparse vector for every observation where only\n",
    "# the class label is 1, and every other class is 0.\n",
    "# more info here:\n",
    "# https://www.tensorflow.org/versions/0.6.0/tutorials/mnist/download/index.html#dataset-object\n",
    "mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(55000, 10000, 5000)\n"
     ]
    }
   ],
   "source": [
    "# %%\n",
    "# mnist is now a DataSet with accessors for:\n",
    "# 'train', 'test', and 'validation'.\n",
    "# within each, we can access:\n",
    "# images, labels, and num_examples\n",
    "print(mnist.train.num_examples,\n",
    "      mnist.test.num_examples,\n",
    "      mnist.validation.num_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((55000, 784), (55000, 10))\n"
     ]
    }
   ],
   "source": [
    "# %% the images are stored as:\n",
    "# n_observations x n_features tensor (n-dim array)\n",
    "# the labels are stored as n_observations x n_labels,\n",
    "# where each observation is a one-hot vector.\n",
    "print(mnist.train.images.shape, mnist.train.labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.0, 1.0)\n"
     ]
    }
   ],
   "source": [
    "# %% the range of the values of the images is from 0-1\n",
    "print(np.min(mnist.train.images), np.max(mnist.train.images))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fc304bdf750>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnV+IfNlV77+r629X//v1hDszkNHxinAvCDIoBi5z4UYU\nbxBhxIfcEB+SK4gPRgVfEn2ZV+NDIFzwJY5hFINXhdwZXzSRIJcIXodr5jrRiREuMxrN/DLj7/fr\n7uru6urq2j50rdPrrNr7nNO/rqo+Xef7gc05dbq6and1fc9ae+2115YQAgghzWLjtjtACFk9FD4h\nDYTCJ6SBUPiENBAKn5AGQuET0kBuJHwR+ZCIfENEvikin1xUpwghy0Uedx5fRDYAfBPAjwL4FwCv\nAfhICOEb7nlMFCDklgghSOz6TSz+BwD8Qwjh7RDCOYDfB/DCDV6PELIibiL89wP4J/P4W7NrhJCa\nw+AeIQ3kJsL/ZwDfbR4/M7tGCKk5NxH+awC+T0SeFZEugI8AeHUx3SKELJP24/5iCOFCRD4B4Eu4\nvIG8FEJ4c2E9I4Qsjceezqv8BpzOI+TWWMZ0HiHkjkLhE9JAKHxCGgiFT0gDofAJaSAUPiENhMIn\npIFQ+IQ0EAqfkAZC4RPSQCh8QhoIhU9IA6HwCWkgFD4hDYTCJ6SBUPiENBAKn5AGQuET0kAofEIa\nCIVPSAOh8AlpIBQ+IQ2EwiekgVD4hDQQCp+QBkLhE9JAKHxCGgiFT0gDofAJaSAUPiENhMInpIG0\nb/LLIvIWgAMAUwDnIYQPLKJThJDlciPh41LwHwwhPFxEZwghq+Gmrr4s4DUIISvmpqINAL4sIq+J\nyM8tokOEkOVzU1f/+RDCt0Xk3+HyBvBmCOGri+gYIWR53MjihxC+PTu+C+CLABjcI+QO8NjCF5GB\niGzPzrcA/DiAry+qY4SQ5XETV/8pAF8UkTB7nd8LIXxpMd0ihCwTCSEs9w0ubwyEkFsghCCx65yK\nI6SBUPiENBAKn5AGctN5fFIBESl8XHZ9GZT1ycZ+rhsHEpHs9crOYz8LIWTv6c/L+qPP97/vrzcd\nCn8JVPnSF11bRf9SrUgwVUQjItjY2Mhaq9XKPS67BgDT6RTT6RQXFxfZubaY+O25/l6qlf1+U6Dw\nl4QKaWNjo1BosbbKfvkjcCkEFZkV3HQ6LX19FXG73Ua73a50bh8DwGQywcXFRfSofUh5ARcXFzg/\nP482ALmbh/6eveE1BQp/CVhxpQSWOq5C+N7aeosbs7Te4qZotVrodDrR1u12S68BSAr3/Pw860PK\nnZ9MJjg7O8N4PMbZ2RnOzs5ynoT+jvdumiZ+Cn/BeNGnWuo5qxB+q9XKhK7nMeGre2zPqwi/1+uh\n1+uh2+3mjv489jMAmWjH4/Hc+WQySQ5DQgg4Pz/H6ekpTk9P0Wq1MkFbjyE2bGmS6AEKf2n4G4AV\nWmq8q1/UZWLFri62PQeuXG3rZut5mUDa7Tb6/X7WNjc3c4+Lfra5uYkQAkajEc7OzjAajbKmj714\ntem18XiMbreLdruds/STyQTj8Tj6+TZN9ACFvxRioi+ysvbxKoRvx9Z+nA1cCt82Fb0dY6fodDrY\n3NzEYDDAYDDIzmPXYo9DCJnF1nZycpKde3ffxh/0ppESvX7eNl6hbr6eNwUKfwnYQF1M/N7axtzt\nZbGxsYFOp4N2u5076jlwOcaeTCa54/n5OVqtVqnwu90uBoMBtra2km17ezt5LYSAk5MTHB8fZ0d7\nPh6Po0FHPT89Pc3ErYG+s7MzdDqd7POdTqdzsyhNEj1A4UepMt8ce6xCj0WvY8fY+bItvr5PTPSd\nTicLkHnR63kV4RcJPHbdPtbX0KYBQD1PCV+PvV4PwNXMhPVWxuMxACRjF00K8FH4Dg1+pcbgZdeK\nhO6tfOzxbbr6djot1aq4+kWuvXX9NaCnrrkKTz/HTqeDXq+XCVtESl39TqcTDeDp59rtdnM3NG36\n2hR+Q1FBF4nDN7WY17HsMTd/Va6+74ftD4BoUO9xgntFgb1+v49er5dzwYGrG2+73Ua3281uNHpd\nxRkL7KnwY31Uj6zb7WbTfHaqTz2dpkDhz7Cuu52Ltu6mdTtTj/WLfB2xr1L4djovFly0LvIyp/Ps\n56afjf4P9MZrLb1e0z6kmk4J+r9ZW7vdxunpae49NR6wypTp24bCRz5N1lucXq+XWSfb9Jr9Wb/f\nTwrfP44F/FYl/KIUWuAq7fVxEng0eBhL1vHNeks2h0FFrqK3/4+LiwsA6Zz8brc710cbf9H3VNGX\nTfWtK40Xfmyxilr8brebc1k3NzdLm7VgVZu9CSxb+ADmEohSKbux4FkZalVjw56yG6Htn8Y7bDAy\nla9vj2rxrQcXS5gC5qf6KPyGEBO9tzAqeJ2islNVsfNer3dtsdvHq8rVL1ojcJOFOmUZi0U3He2D\nfg7W8scCdrFz6+qnFgype6+iH41GFH5TsW6md/U3NzezaSdtOzs7uaOelwnfW/dVC9/+ranzMnFV\nee2yY9lrxNKXq2Td9Xq9uYVI9gYA5C39aDTKuf5NoZHC9/Pu9rFaebXgKmjbYsLXoxV+TNipsfV1\ncvWLxFjkBtu/3x+rCLPqjSHmIfg029TjsnUOKU/Fnvf7/dxUnZ2OHI/HOD09zWYU/KxCU2ic8O08\nfcwSb25uYmdnB7u7u9GmySbezdcgX7fbLQye2ekjDaDFvrwpUkGt1BSXPwLz7rh3vWM3AT8UKHrf\nmzQbI0ilFRfdVH2fvcVP3UCaRiOFb6O7fi5+MBhgd3cXe3t72Nvby871uL29HQ3q9fv9LJIdCyZZ\n78KKBLi+O+3F7AVup+D80U/n+YCbFU/Mqur7x1JmbQENmwcQe+ynCfWxnRWINT8boOe2f17UVT2I\nJtFI4dtIsf9ibW1t5YS/t7eHe/fuZefqzvvpPBW+RqP9lyoWNU9Fz4sos6qpVXV6VIsay9e3c9up\nVnZzUZfautg+99/3yZ63Wq1cgk8s6cf/zwDMuer+c68i+CaJv3HCB66mi/w8fb/fx/b2dubWq+j3\n9/dx79493Lt3D4PBIGmJut1uVDgAcsIBrlx9b/HKpszsEMFbzpjw/FHjGD4HXs+t1Y8FyDQanipt\ndX5+jvF4nB1j50U3h1arlVy5Z1N9+/1+9lnZ6L/9rIvc/CZbe6CBwtd/up+n16bjey/6/f197O/v\nYzAYzLma1uW0lidmTax7b4VaNRfeu8cxq27FpgEuPReRuWQk22wyTWz8r8KPLdlVAduUWN9sn2Kt\n3W7PzaBsbW3h7Owse44u3wWuRB/LKiy6AaSChE2hkcK3rr5O12mwzgrfiv+JJ57AE088gc3NzbmA\nUtF0XCrqbgVsxVlF+DHR6bkKz1ausceNjY1kLv1kMknGKLSFEOZW7NmjL6Ch7fT0NCuoYT0A21T4\nu7u72f9Bf+/8/HwubdgOW3q9XtTiL2IWZR1ppPBjrr6dulNXX917L/yy6SQgXSLaBuSsWFWYmpKa\nYjqdFrrL+jq2ao09bmxsZDe60WiEwWCQlbSaTqeZ8FNTj/r+3qOw720LZ/jz0WiUvCmdnZ2h2+1m\nN9zT09PsRmFz9PXztv9Db/GruPu0+A3DirVo6i02zgXmp9SU1BSbFbtaxVQrWyGmFj8lfk1KSQlf\nROZEZ5ufjvQ3gBBCdAhhhW8Ffx3h2/e3lly9ExsctHGNWEafD576gKefmagym7JONFL4+g/3VlJX\nbh0fH2dBL5vcMZ1O0e/3owK3j1Nz6DZjzI557XkVi19ljF3k6tsgnAp1c3MTx8fHmcVPNfu5xdx9\nTZBJufvWrU+573aO3s6+2BV+PvlGb85W6PpZ6HufnJxkNyK9AWnfq8yorBONE76djvLC18U59ktl\no/EXFxfZWNJnnsWsTKyp8GMVZKsKv2h+3FtifxSR3N/sg3udTmcuoGePPrjnb0L2Rpa6ufmpPW91\nfa6FFb3Nl9Cgqh+rW+Hbm5uW8dIhhH4m2ocm0Ujhx8bXajlshph17fX5WhwiNQ9vLU7s3ArTB7aq\nCN9OA/p5dCv+1JSeiGRiiOUx2AUysaFObBrS5wykpvFU9P751tpai29nS3QGxt6gYqv7rGel7xmz\n+PYmZG8+TaFU+CLyEoCfBHA/hPADs2v7AP4ngGcBvAXgwyGEgyX2c6H4iHoqkGUTVvSL1Ol0kmvV\nU/Pr9rEPhvnzKsK/TgKPHw6ISDJ5x3s5y0jg8c+3DZh39e3Nybr6dvmzvzGlLL4KX4cedPWL+TyA\n/wHgd8y1TwH4sxDCb4jIJwH86uxa7bHC0S+jT1O11sNbsVarlUyeKbKE9lrZTjFV+l81Zdf3EUDp\nuvjYbIV3pVOeTiyQ5ltqiGQ/f+t9aZAvNsZPufo+2Bmz+HbGgK6+I4TwVRF51l1+AcB/mZ2/DODP\ncUeED+QF7VNpY8+xLmOr1SpMoCmytvb1Uscqwtdj0UKZ1DE2fo9NcwHxVXxF7xu7Cfhr/jP26Ptb\ni58K7sVuWCmLPxqNosE9uvrX48kQwn0ACCG8IyJPLrBPS8UH96zoU3Ps+sXp9/tZVLwoiaZq80tG\nq2Tu2b+j6Dx1VMqsun1e0fun3jt17t8v1vwYP+bqW+HH1kHExvhan98H9+jqPz536hOz4o/NzVvr\n5SP/IlIo+OuIP+YVNOHL5z0OIJ5tZ+v+X1f4ftoxljZsrT1d/WrcF5GnQgj3ReRpAN9ZZKeWScwt\ntTcAAHNuo/cQysbyMVEXJZA0zdoosVkDa+1TqyiLRB+Lc8RuzvZ/0MTPvqrwZdaUVwF8HMCnAXwM\nwCuL7dZyseNdG0W3rqgXvVp9TRKJRe5jAT7f/HN8EtC6U+Te21kVK3ptdgOOMmsfy3coEn6T/gdA\ntem8LwD4IID3icg/AngRwK8D+EMR+VkAbwP48DI7uWisZfD59YoXvVoZK/zUdF5smiv1O021OEqR\n+P08vs01SM1CpGYZ/A2g6Va/SlT/o4kf/diC+7Iy9MthH4cQssw0+2XRL6AtwZyKVpdFtIsi3U2z\nODGLn0rX9cKPpRID81OdVQXfpM9daWTmnnXl9Zp37VutFiaTSS65p9Vq5Z7vZwJSc+tF500VvR5j\nFt9a+5j4Y0MFYP7/SPGnaZzwgflpJ7XiKQtkj/Y1YnPq/nHV5zWNmOB1/j7l6usY37+OctMxfpNo\nrPCt5VeKElfs0X9JrvO47LlNosjix4J7avFTN1A9p/jLaaTwFYpw9ahVT+02bKsC+c1INT03FlDV\no13/rzUIYnP3TRY90HDhk9Vg3XG/S5HNyut2u9jf38+qGet+BXbVYAghl5jjqwAdHh7iwYMHePTo\nEQ4PDzEcDrOFObZun82UZFSfkAViBa/nmpFnK+vYo5Yy39nZwdbW1lzZcs3I89WF9Pzg4CAn/KOj\no2wNvi60irn7TbP6FD5ZCqn4yMbGVYVj3ZPQbkh67969bMeiwWCQrb33Ft8vtdU8fCv8g4ODzOL7\n+n2pIiBNgcInS8NP2wF5i69FP+2+hHbjkiJXX0t8DYdDHB0dZe3g4ACPHj3KufrHx8fZajytZOxj\nAxQ+ITfEu/hW+HaMr9WNreB1u7KdnZ1M+D5r0lr84XCIw8PDzMJrOzw8zFx9b/FjiVRNs/oUPlkK\nsQQbnbKz+xlY4Wtgz7r63uKr8MfjMU5OTjAcDnFwcICHDx/iwYMHODg4yKz/cDiMBvdSiVRNgsIn\nK8HO1acsvo3o212IY8K3BTQPDw/x8OFDvPfeezg4OMiKaurY3wf3FDv33zQofLJwYhl5evRbl2lg\nz+5ZqHvk2Yi+LX5qV0tqgE/H+oeHh7l6/rHaeoTCJwtGRZ/aw95P4cU2xbQJPH5vA7/m3hdKsdVz\n7ZRdE935Iih8snBsBR1fVMPu22ebndKz22HHaufbCjte+H5X3qZG7cug8MlCsRZfi2XatFwvfGvt\ntflqO76YZhWLH8vOI1dQ+GTh+PLYKmAd25eJ398srKsP5PdFiNXVS1l8Wv0rKHyyUGIW3y6rjbn5\n3tUvqvdfZPHV6tsKxnaMT66g8MnCKdoQIzbG98G92C69sdp6MVffBvZ8FV2K/woKnywUb/F9XXzv\n6sfG+D7pJ1Zsw66v92P8WIFTij4PhU8Wji+VHSuq4ZsN5hWVNrPj+FgkX1NyfWFTCj8PhU8WTlHp\nbJ/YE7PoReXJU+vsY269D+hR/FdQ+GSheBfdb5aREr0dv2tmXmxnYc3Ft8L3+9zHxE/R56HwydIo\nsvr+pqBY4dvSWdq8xffr7P3KO4o+DoVPFk6Rq5/amVex6+11IY5tuuTWrrP3wo+JnuLPQ+GTpVE2\n1o9F7q3FtyvwVOiPHj1Kuvp211u6+cVQ+GThlFl87+Zbq2+3LbPFNmyFnTJXP1V6m1xB4ZOlkAry\nFUX0NTPPCl/X1B8dHWVVdnxU3wb3YpugUvjzUPhkoVhx++2v7Kq72FbXap11jD8ajTJXX9fax6z9\neDxOJupQ9HEofLIwbNKOzdSzhTa0bPbm5ma0gq4ut1WLPxqNMquvLr9W17EpurF8fIo+DYVPboSP\nyotILj9f03K3t7exs7OD3d3dqPCB/PjeRvWtu2+tvbr5MeFT9MVQ+OSxSW2Yoavy1OKr8NXib29v\n54RvXX3NwVfhe3f/6Ogom9qzS3Ctm0/Rl7NR9gQReUlE7ovI35hrL4rIt0Tkr2ftQ8vtJqkbMdFb\nV99umuGFr8tvvatvF994i68Vc3VaL2XxKfpqlAofwOcB/NfI9c+EEH5w1v5kwf0id4TYjjnW1bdj\n/CJX30f0Nbhnx/jq6qvwy8b4JE2pqx9C+KqIPBv5kUSukQYRE7139WMWX6P7fowfc/XtGF+De7bq\njk/cIdWoYvFTfEJEXheR3xKRvYX1iNSelJtfRfhFFt8G92JR/ZTF1zl8Cr86jyv83wTwvSGE5wC8\nA+Azi+sSqTt+rb1dT19WbMPXy7eVc9XS26aLc9Tt19x8vyKP4/vr8VhR/RDCu+bh5wD88WK6Q+qO\n3Q1HhW+LbWjUfmtrK7PsehPQ5B19DeBqUc7FxQU2NjZyK/F80czUenuK/vpUFb7AjOlF5OkQwjuz\nhz8N4OuL7hipJ1pWy2bl2fPt7e3cFlhaV8+O6RW7ik7xFj1WJrvJm10uilLhi8gXAHwQwPtE5B8B\nvAjgR0TkOQBTAG8B+Pkl9pHUCCt8697rUS2+bnpp3Xt9no7lrYD1mhV+yuJz2e3NqRLV/2jk8ueX\n0BdyB4htfGlbmavf6XQwmUxyordVcb3FT1XLpZt/M5i5R66FWnx17W3kfjAY5ISfsvgAsp1tYmm6\nqR1xuOx2cVD45FrELL7uc69JOn6Mby1+u93OrDwwX3En5urbJbep1FyK/3pQ+ORa+DF+r9fLbXXt\nXX0Vvwb3Wq0WJpNJFtW3q/Fi03V+Kywm6iyGmyTwkIZiLb5fiGOtvV9/79fe+0o7mqhjF+Fw19vl\nQItProVN3mm327n98GLj+aqC15V3RWW1KPrFQYtProWfx/dW37r2fn97oLiYplbY0WIbfrMMCn9x\nUPjkWtjgnhW+jexXtfi6Ak/z8A8ODnBwcJCz+MzFXw4UPqlMbCGOjez76bvY/vYpV7/I4lP4i4dj\nfFIJv/S2qqtfZYxvLf7h4WFWU8+P8cnioPBJKX4Zrl+V5y2+dfXV4lfZMOPw8BAHBwe5nXO00AYj\n+ouFwieFxIppqsW3U3pFY3wN7vk19z6qrxbf75dHV3/xUPhkjtimlto0EcdO49mma+19SS1fXccX\n0zw5Ocmaz9qjxV88FD7JYV35WNMqOpqhF5u7j03fjcdjTKfTrEim3wnXFt/w+fnMx188FD7J4Rfh\n+OaF73Pyu91uFvVX114ttq6+89V0/A2gaEUeWQwUPslhha8uvY7je70e9vb2sLu7GxW9XX1nS2rp\nSjwRSYo+tf7eip8sDgqf5LDCtxF7bWrtiyy+L7Bhi2dYV98uyLHi1wAgF+YsDwqf5PAWX5fc6go8\ntfhFwtcgnq2sYwttpFx8uytO7MZB8S8OCp/ksLn4dppOhW6Fn3L11WoD+fX24/E4avG9+G1lHYp+\nOVD4JIe3+HYnHG/tYxbfFtqwwT0/fVcU1VdYT295UPgkh192axN0/NLbsv3urZuvCTtFU3c6PCDL\nh4t0yBwqfrtphp3is0ttbZIPgJyL7sf3fturWDktshoofBLFi1/Frk1vBlb8ihW9tfi6FDc1T09W\nB4VPcth0XWvxY8L3ordW34tfRe+LZ9rMPIp/dVD4ZA4vfhW7Hcdbi39dV58W//ah8MkcqTG+F78+\nx7v6flGOd/U5xr99KHySw67EK3P1vcW3K/FiFr/K7jhkNVD4ZA5r8W1wz0b0U+N84GqMHwvuxdx9\nin71UPgkh7f2dnyvm2R64fsxvrf2fnssuvu3D4VPcnjR27p6tvlAnxe+F7xm7NmsPSt+in61MHOP\nAJgvpunLa9mimqlMPSDv5hcJXzP3uEvO7UCLT+aKaXqLb/P2fYquracH5C2+z9HXyrkxi89U3dVS\nKnwReUZEviIifysib4jIL82u74vIl0Tk70XkT0Vkb/ndJYvGi95H8+3Y3rr6sdLZwFWpLbX4mqMf\nc/XtZhm0+KulisWfAPiVEML3A/hPAH5BRP4jgE8B+LMQwn8A8BUAv7q8bpJl4EWvx1hgTy1+VVc/\nZvG1qGYswEeLv1pKhR9CeCeE8PrsfAjgTQDPAHgBwMuzp70M4KeW1UmyeFKiTwX3rMVfxBjf1tfj\nlN7quVZwT0S+B8BzAP4SwFMhhPvA5c1BRJ5ceO/I0vF59kVRfR3jx+byY8K3Fl+tfdEYn8JfHZWF\nLyLbAP4IwC+HEIYi4v9L/K/dEXytfNt8hp6tsKuWX28Mdlssa+XV0tv6+Wr1/fjelugiq6OS8EWk\njUvR/24I4ZXZ5fsi8lQI4b6IPA3gO8vqJFkcfgGOb7EAnl2HH9sOS4U7mUwygXuh+1JbzN67XapO\n5/02gL8LIXzWXHsVwMdn5x8D8Ir/JVJPUll5XvTa7NRdbANMH8RLid5bexvRp8VfLaUWX0SeB/Az\nAN4Qka/h0qX/NQCfBvAHIvKzAN4G8OFldpQshlRKrp7bufqUxbdpubYg5nQ6LRX9aDTKldqya/LJ\n6igVfgjhLwC0Ej/+scV2h6wCv+rOCttuoOHFr8LXLavtnL026857917P1UuwZbjp6q8Wpuw2DGvx\nYwtwvKvv6+xZi+/n7O0Yv8jqq3tv3Xy6+quFwm8gdrmtjdjbKbsii29r5vsVeDEr76/Fhgi0+KuF\nwm8YavGtq59KyY1Z/Ha7jclkAiCfpWen7VKC18f6u7EjWQ0UfgPxNfPt/ni6SYbdHadoKi+Wj2/F\n7ve61/gAuV0o/IZhXfx+v5/tiaeCv3fvXrYb7mAwyKy/it6P663oufru7kDhNwx189Xa6754Ozs7\n2N3dnRN+v9/PLL7d+trn4mtKroqfq+/qDdfjNwxNy7UWf3t7OxN9TPgxi2+Fr2N3Fb4+tptn0OLX\nC1r8hmEtvhX+3t4e9vf3c8Lf3NzMxvgx4ftFOLbR4tcbCr9hpIRvLb7uhBuz+DYv3y+7tRbfL7ul\n8OsFhd8wUq6+tfh+V9zUGD+27NYK3+fjk/pA4TcMjerb4J5a/P39fezt7c0l8qRc/ZTFt64+LX49\nofAbhl2g4zP2dC4/tirPrsjzGXuxOXxfWouirxcUfkOIFdX0JbdSzZPaLCNVaIMWv35Q+GtOSvDX\nFbvio/rXET6pDxT+GuPr6el5ytr730ndBFJTejHhW2tPi18fKPw1JSZkPZZZei94ex6z+Hac76fy\naPHrCTP31pgyK17V+ntirr5fkmt3xWVNvfpB4a8hRTXzq47vU0cVcCqJh1V07wZ09deUoqCev1b0\nnBhq8WNz+Zqjb5N3aPHrB4W/psQsua21V9bsvvd+jK9Ha/1ti1XVoejrBYW/Znih+2br69myW7FK\nO/4mYF/flu/ym2/Ym8HFxUXpFCFZPRT+GmKtuz+mBJ8qrKkCjw0LfLVebSr41O+T24fCX0NiFtlv\ne10k/pTLH3t9X6230+lkgT+Kvr5Q+GuGijJlkVXc1t33G2dYS23Fq6+fsvjqLUwmk+zof5/UAwp/\nDbGitNbY1tC3QvfX7I0jFeSzHoV/H03cSXkM5Pah8NeQVPAtZuVjN4Gy/H19HBO9jw9Q/PWEwl8z\nitxwb91TR30d+5qx109F9s/Pz+c22KTo6wWFv4ZY4fudcP1OOTEr7effbbPpuDYl126H5efxmbxT\nPyj8NUNdcOvaa5GNfr8/V0uv2+3OFdrwe9tZYR8dHeHw8BBHR0cYDoc4Pj7Omq2rH0vZpfjrA4W/\nZqi1V7fbVtfRzTNU+HaXnI2Ny2Ubaq1VtHrU86Ojo6wNh8Os+bJbfmkuRV8vShfpiMgzIvIVEflb\nEXlDRH5xdv1FEfmWiPz1rH1o+d0lZcSEr6K322P50tm2pp6uvLO19IbDYWbt1eJ7q5+y+KzAUz+q\nWPwJgF8JIbwuItsA/q+IfHn2s8+EED6zvO6RxyG2N55a+yKLb4XvN8vQVib609PTOU+B6/HrR6nw\nQwjvAHhndj4UkTcBvH/2Y4Zqa4YP7BUJ34/xgfzW1yp8deNPTk5yFl/dfC9+jQn4wB8tfn241np8\nEfkeAM8B+D+zS58QkddF5LdEZG/BfSOPgXf1/cYZukNOkcXXMb519Y+Pj+cCe9bq2zF+bNNMRvbr\nRWXhz9z8PwLwyyGEIYDfBPC9IYTncOkR0OWvAVb46urrmD7l6qfG+NbiD4fDzNpXierH6uqT+lAp\nqi8ibVyK/ndDCK8AQAjhXfOUzwH448V3jzwONtsutnoutvJO8dV1vNW3Ft7vk6cBPQutfD2pavF/\nG8DfhRA+qxdE5Gnz858G8PVFdow8PjZpxo63bfMJN1agsdLZfous2G64XuQUfX0ptfgi8jyAnwHw\nhoh8DUAA8GsAPioizwGYAngLwM8vsZ/kmvhiGF78PujmK+qktsGO7ZYTc+Up+npTJar/FwBakR/9\nyeK7Q24qubh7AAAEQUlEQVSKL4flRW/r4NlyWfb3UzXz1eqXbYNN0dcfVtldQ1Li93Prsc0uUhV0\nreh91N66+hT93YDCX0Ou4+bHxvkpi++3wbYWn2m5dwsKf82IVb9NBfeKLL4vna0RfG6DvR5Q+GuI\nD9J5q19k8b2r76f0UlthU/h3C67OW0Ni+9f7AJ2txmPn9s/Pz6MpuXbe3s7Z28w8cneg8NeM2CIb\nm4vvn3N2dpYl5xweHmIymeC9997L2oMHD3BwcIDhcJgUPdfb3z0o/DXDp9xqhh4wv/WVegCajru9\nvY2Liws8fPgw11T4qQ0xKfq7B4W/Ztgx+ng8ztJx9Zq6/zYHX/P3B4MBLi4ucjn56vYfHx9nQb1Y\nBiCn8u4WFP6aYd14u8NtLEp/fHycLdvVNp1O5xbe6LkK3wYMbaCQ3B0o/DXDuvr6WK1/p9PJ3Htf\nXVfPQwi5wht2F9zRaDQ3rqerfzeh8NcMu55ehRnbJdduq6Xn7XYbIYTcGD5WUde69n4qkNwNZNn/\nLBHht2HFpDbEKGp2ea4vk+1LZgNIHkm9CCFEq2TR4q8htL6kDGbuEdJAKHxCGgiFT0gDofAJaSBL\nj+oTQuoHLT4hDYTCJ6SBrEz4IvIhEfmGiHxTRD65qvetioi8JSL/T0S+JiJ/VYP+vCQi90Xkb8y1\nfRH5koj8vYj86W3uXpToX202Uo1s9vpLs+u1+AxvezPalYzxRWQDwDcB/CiAfwHwGoCPhBC+sfQ3\nr4iI/H8APxRCeHjbfQEAEfnPAIYAfieE8AOza58G8K8hhN+Y3Tz3QwifqlH/XgRwVIeNVGf7Pjxt\nN3sF8AKA/44afIYF/ftvWMFnuCqL/wEA/xBCeDuEcA7g93H5R9YJQY2GPiGErwLwN6EXALw8O38Z\nwE+ttFOGRP+AmmykGkJ4J4Tw+ux8COBNAM+gJp9hon8r24x2VV/09wP4J/P4W7j6I+tCAPBlEXlN\nRH7utjuT4MkQwn0g28X4yVvuT4zabaRqNnv9SwBP1e0zvI3NaGtj4WrA8yGEHwTwEwB+YebK1p26\nzcXWbiPVyGav/jO71c/wtjajXZXw/xnAd5vHz8yu1YYQwrdnx3cBfBGXw5O6cV9EngKyMeJ3brk/\nOUII74aroNHnAPzwbfYnttkravQZpjajXcVnuCrhvwbg+0TkWRHpAvgIgFdX9N6liMhgdueFiGwB\n+HHUYxNQQX689yqAj8/OPwbgFf8LKybXvxpupDq32Svq9Rne2ma0K8vcm01LfBaXN5uXQgi/vpI3\nroCI/HtcWvmAy6XKv3fb/RORLwD4IID3AbgP4EUA/wvAHwL4LgBvA/hwCOFRjfr3I7gcq2Ybqep4\n+hb69zyA/w3gDVz+X3Wz178C8Ae45c+woH8fxQo+Q6bsEtJAGNwjpIFQ+IQ0EAqfkAZC4RPSQCh8\nQhoIhU9IA6HwCWkgFD4hDeTfAKZ6ORNIn8/jAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc304cd6c10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# %% we can visualize any one of the images by reshaping it to a 28x28 image\n",
    "plt.imshow(np.reshape(mnist.train.images[100, :], (28, 28)), cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %% We can create a container for an input image using tensorflow's graph:\n",
    "# We allow the first dimension to be None, since this will eventually\n",
    "# represent our mini-batches, or how many images we feed into a network\n",
    "# at a time during training/validation/testing.\n",
    "# The second dimension is the number of features that the image has.\n",
    "n_input = 784\n",
    "n_output = 10\n",
    "net_input = tf.placeholder(tf.float32, [None, n_input])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %% We can write a simple regression (y = W*x + b) as:\n",
    "W = tf.Variable(tf.zeros([n_input, n_output]))\n",
    "b = tf.Variable(tf.zeros([n_output]))\n",
    "net_output = tf.nn.softmax(tf.matmul(net_input, W) + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %% We'll create a placeholder for the true output of the network\n",
    "y_true = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %% And then write our loss function:\n",
    "cross_entropy = -tf.reduce_sum(y_true * tf.log(net_output))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %% This would equate each label in our one-hot vector between the\n",
    "# prediction and actual using the argmax as the predicted label\n",
    "correct_prediction = tf.equal(\n",
    "    tf.argmax(net_output, 1), tf.argmax(y_true, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %% And now we can look at the mean of our network's correct guesses\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %% We can tell the tensorflow graph to train w/ gradient descent using\n",
    "# our loss function and an input learning rate\n",
    "optimizer = tf.train.GradientDescentOptimizer(\n",
    "    0.01).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %% We now create a new session to actually perform the initialization the\n",
    "# variables:\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8994\n",
      "0.9236\n",
      "0.9242\n",
      "0.9152\n",
      "0.9216\n",
      "0.915\n",
      "0.918\n",
      "0.9256\n",
      "0.926\n",
      "0.9228\n"
     ]
    }
   ],
   "source": [
    "# %% Now actually do some training:\n",
    "batch_size = 100\n",
    "n_epochs = 10\n",
    "for epoch_i in range(n_epochs):\n",
    "    for batch_i in range(mnist.train.num_examples // batch_size):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        sess.run(optimizer, feed_dict={\n",
    "            net_input: batch_xs,\n",
    "            y_true: batch_ys\n",
    "        })\n",
    "    print(sess.run(accuracy,\n",
    "                   feed_dict={\n",
    "                       net_input: mnist.validation.images,\n",
    "                       y_true: mnist.validation.labels\n",
    "                   }))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.923\n"
     ]
    }
   ],
   "source": [
    "# %% Print final test accuracy:\n",
    "print(sess.run(accuracy,\n",
    "               feed_dict={\n",
    "                   net_input: mnist.test.images,\n",
    "                   y_true: mnist.test.labels\n",
    "               }))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n# We could do the same thing w/ Keras like so:\\nfrom keras.models import Sequential\\nmodel = Sequential()\\n\\nfrom keras.layers.core import Dense, Activation\\nmodel.add(Dense(output_dim=10, input_dim=784, init=\\'zero\\'))\\nmodel.add(Activation(\"softmax\"))\\n\\nfrom keras.optimizers import SGD\\nmodel.compile(loss=\\'categorical_crossentropy\\', \\n    optimizer=SGD(lr=learning_rate))\\n\\nmodel.fit(mnist.train.images, mnist.train.labels, nb_epoch=n_epochs,\\n          batch_size=batch_size, show_accuracy=True)\\n\\nobjective_score = model.evaluate(mnist.test.images, mnist.test.labels,\\n                                 batch_size=100, show_accuracy=True)\\n'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# %%\n",
    "\"\"\"\n",
    "# We could do the same thing w/ Keras like so:\n",
    "from keras.models import Sequential\n",
    "model = Sequential()\n",
    "\n",
    "from keras.layers.core import Dense, Activation\n",
    "model.add(Dense(output_dim=10, input_dim=784, init='zero'))\n",
    "model.add(Activation(\"softmax\"))\n",
    "\n",
    "from keras.optimizers import SGD\n",
    "model.compile(loss='categorical_crossentropy', \n",
    "    optimizer=SGD(lr=learning_rate))\n",
    "\n",
    "model.fit(mnist.train.images, mnist.train.labels, nb_epoch=n_epochs,\n",
    "          batch_size=batch_size, show_accuracy=True)\n",
    "\n",
    "objective_score = model.evaluate(mnist.test.images, mnist.test.labels,\n",
    "                                 batch_size=100, show_accuracy=True)\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
